{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ef92c15d",
   "metadata": {},
   "source": [
    "## Metrics\n",
    "\n",
    "汇总常见2分类的指标，例如: AUC，ROC曲线，ACC, 敏感性， 特异性，精确度，召回率，PPV, NPV, F1\n",
    "\n",
    "具体的介绍，可以参考一下：https://blog.csdn.net/sunflower_sara/article/details/81214897"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da73b59a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "from onekey_algo import get_param_in_cwd\n",
    "\n",
    "os.makedirs('img', exist_ok=True)\n",
    "os.makedirs('results', exist_ok=True)\n",
    "subsets = ['train', 'test']\n",
    "sel_models = ['ResNet50', 'ShuffleNet']\n",
    "label_file = r'C:/Users/onekey/Desktop/demo/group.csv'\n",
    "model_root = r'models3d/'\n",
    "group_info = pd.read_csv(label_file)\n",
    "group_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58a846b4",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy  as np\n",
    "import re\n",
    "from onekey_algo.custom.components import metrics\n",
    "from onekey_algo.custom.components.comp1 import draw_roc, normalize_df\n",
    "from matplotlib import pyplot as plt\n",
    "from onekey_algo.custom.utils import print_join_info\n",
    "\n",
    "\n",
    "def get_log(log_path, map2gz:bool = True):\n",
    "    log_ = pd.read_csv(log_path, names=['fname', 'pred_score', 'pred_label', 'gt'], sep='\\t')\n",
    "    if map2gz:\n",
    "        log_['ID'] = log_['fname'].map(lambda x: os.path.splitext(os.path.basename(x))[0] + '.gz')\n",
    "    else:\n",
    "        log_['ID'] = log_['fname'].map(lambda x: os.path.basename(x))\n",
    "    return log_\n",
    "\n",
    "def map_mn(x):\n",
    "    return x.replace('densen', 'DenseN').replace('resnet', 'ResNet').replace('vgg', 'VGG').replace('inception_v3', 'InceptionV3')\n",
    "\n",
    "all_log_ = []\n",
    "metrics_dfs = []\n",
    "for epoch_ in range(0, 5):\n",
    "    metric_results = []\n",
    "    all_preds = []\n",
    "    all_gts = []\n",
    "    all_model_names = []\n",
    "    for model in sel_models:\n",
    "        all_pred = []\n",
    "        all_gt = []\n",
    "        all_groups = []\n",
    "        # 评估的事viz的最佳非训练集表现的模型\n",
    "#         val_log = pd.concat([get_log(os.path.join(model_root, model, f\"viz/BST_TRAIN_RESULTS.txt\")),\n",
    "#                             get_log(os.path.join(model_root, model, f\"viz/BST_VAL_RESULTS.txt\"))], axis=0)\n",
    "        # 指定epoch评估的模型\n",
    "        val_log = pd.concat([get_log(os.path.join(model_root, model, f\"train/Epoch-{epoch_}.txt\")),\n",
    "                            get_log(os.path.join(model_root, model, f\"valid/Epoch-{epoch_}.txt\"))], axis=0)\n",
    "        val_log = pd.merge(val_log.drop_duplicates('ID'), group_info, on='ID', how='inner')\n",
    "        val_log['gt'] = val_log['label']\n",
    "        val_log['model'] = f\"{model}\"\n",
    "        ug_groups = subsets\n",
    "        ul_labels = np.unique(val_log['pred_label'])\n",
    "        for g in ug_groups:\n",
    "            sub_group = val_log[val_log['group'] == g]\n",
    "            sub_group['label-1'] = list(map(lambda x: x[0] if x[1] == 1 else 1-x[0], \n",
    "                                            np.array(sub_group[['pred_score', 'pred_label']])))\n",
    "            sub_group['label-0'] = 1 - sub_group['label-1']\n",
    "            sub_group[['ID', 'label-0', 'label-1']].to_csv(os.path.join('results', f'DL_{model}_{g}.csv'), index=False)\n",
    "            all_groups.append(g)                    \n",
    "            all_log_.append(sub_group)\n",
    "            for ul in [1]:\n",
    "                pred_score = np.array(sub_group['label-1'])\n",
    "                gt = [1 if gt_ == ul else 0 for gt_ in np.array(sub_group['gt'])]\n",
    "                acc, auc, ci, tpr, tnr, ppv, npv, p, r, f1, thres = metrics.analysis_pred_binary(gt, pred_score, use_youden=True)\n",
    "                ci = f\"{ci[0]:.4f}-{ci[1]:.4f}\"\n",
    "                metric_results.append([model, acc, auc, ci, tpr, tnr, ppv, npv, p, r, f1, thres, g])\n",
    "                all_pred.append(pred_score)\n",
    "                all_gt.append(gt)\n",
    "        # 绘制每个模型的ROC\n",
    "        draw_roc(all_gt, all_pred, labels=all_groups, title=f\"Model: {map_mn(model)}\")\n",
    "        plt.savefig(f'img/DL_{model}_roc.svg', bbox_inches='tight')\n",
    "        plt.show()\n",
    "        # 整合到所有模型汇总。\n",
    "        all_preds.extend(all_pred)\n",
    "        all_gts.extend(all_gt)\n",
    "        all_model_names.append(model)\n",
    "    for gi, g in enumerate(all_groups):\n",
    "        draw_roc(all_gts[gi::len(all_groups)], all_preds[gi::len(all_groups)], \n",
    "                 labels=[map_mn(m) for m in all_model_names], \n",
    "                 title=f\"Cohort {g}\")\n",
    "        plt.savefig(f'img/DL_{g}_roc.svg', bbox_inches='tight')\n",
    "        plt.show()\n",
    "    metrics_df = pd.DataFrame(metric_results, \n",
    "                              columns=['ModelName', 'Acc', 'AUC', '95% CI', 'Sensitivity', 'Specificity', 'PPV', 'NPV', \n",
    "                                       'Precision', 'Recall', 'F1',\n",
    "                                       'Youden', 'Cohort'])\n",
    "    display(metrics_df)\n",
    "    metrics_dfs.append(metrics_df)\n",
    "pd.concat(metrics_dfs, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db95e8ec",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
